﻿using HtmlAgilityPack;
using NUglify;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Web;

namespace Test
{
    public class Crawl
    {
        string url { get; set; }
        public Crawl(string url)
        {
            this.url = url;
        }
        public string get()
        {
            var temp = new LinkedList<string>();

            WebClient webClient = new WebClient();
            webClient.Headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36 Edg/100.0.1185.44";
            webClient.Headers["Accept"] = "text/html,application/xhtml+xml,application/xml;";
            webClient.Headers["Accept-Language"] = "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6";
            webClient.Headers["Host"] = "www.baidu.com";
            webClient.Encoding = Encoding.UTF8;
            string html = webClient.DownloadString(url);

            html = Uglify.HtmlToText(html).Code;

            return html.Substring(0,200);
        }
       
public static string ExtractText(string html)
    {
        Regex reg = new Regex("<[^>]+>", RegexOptions.IgnoreCase);
        string s = reg.Replace(html, " ");
        s = HttpUtility.HtmlDecode(s);
        return s;
    }

    /// <summary>
    /// Compiled regular expression for performance.
    /// </summary>
    static Regex _htmlRegex = new Regex("<.*?>", RegexOptions.Compiled);

        /// <summary>
        /// Remove HTML from string with compiled Regex.
        /// </summary>
        public static string StripTagsRegexCompiled(string source)
        {
            return _htmlRegex.Replace(source, string.Empty);
        }

        /// <summary>
        /// Remove HTML tags from string using char array.
        /// </summary>
        public static string StripTagsCharArray(string source)
        {
            char[] array = new char[source.Length];
            int arrayIndex = 0;
            bool inside = false;

            for (int i = 0; i < source.Length; i++)
            {
                char let = source[i];
                if (let == '<')
                {
                    inside = true;
                    continue;
                }
                if (let == '>')
                {
                    inside = false;
                    continue;
                }
                if (!inside)
                {
                    array[arrayIndex] = let;
                    arrayIndex++;
                }
            }
            return new string(array, 0, arrayIndex);
        }


    }
}
